# -*- coding: utf-8 -*-

import requests,json,os,urllib,string,sys,re
import scrapy
from gooooal.sqlHelper import *

reload(sys)
sys.setdefaultencoding('utf8')

class GlqSpider(scrapy.Spider):
    name = 'glq'
    allowed_domains = ['gooooal.com']
    handle_httpstatus_list = [301,403,404]
    num=0
    start_urls = ['http://app.gooooal.com/nflHome.do?method=nflRank']
    def parse(self, response):
        if response.status==404 or response.status==301 or response.status==403:
            print '采集失败404' 
        for i in range(1,3):
            for each in response.xpath('//*[@id="NFL_Atable_'+str(i)+'"]/table'):
                for i in range(2,6):
                    url=each.xpath("./tr["+str(i)+"]/td[1]/img/@src").extract()[0]
                    name=each.xpath("./tr["+str(i)+"]/td[2]/text()[1]").extract()[0]
                    string = name.decode("utf-8")
                    cnobj = re.compile(u'[^\u4E00-\u9FA5]')#非中文
                    cn = cnobj.sub(r'', string)#replace
                    en=each.xpath("./tr["+str(i)+"]/td[2]/text()[2]").extract()[0]
                    enobj=re.compile(u'[^a-zA-Z0-9]')
                    en=enobj.sub(r'',en)
                    if len(en)<1:
                        en=each.xpath("./tr["+str(i)+"]/td[2]").extract()[0]
                        en=en[en.index("<br>")+4:len(en)-5]
                        enobj=re.compile(u'[^a-zA-Z0-9]')
                        en=enobj.sub(r'',en)
                    if en[0:1]==' ':
                        en=en[1:len(en)]
                    if self.save_data(en,cn,'',en+url[len(url)-4:]):
                        self.save_img(url,en)
                    self.num+=1

    def save_data(self,en,cn,big,logo):
        if sqlHelper.selectOne("select id from team where cn=%s and typeText=%s",(cn,'橄榄球')):
            sqlHelper.update("update team set logo=%s,source=%s where cn=%s and typeText=%s",(logo,self.start_urls[0],cn,'橄榄球'))
            return False
        else:
            sqlHelper.update("insert into team(type,typeText,cn,en,logo,big,source) values(%s,%s,%s,%s,%s,%s,%s)",(35,'橄榄球',cn,en,logo,big,self.start_urls[0]))
        return True
    
    def save_img(self,img_url,file_name,file_path='ganlanqiu'):
        img_url=img_url.replace('\/','/')
        try:
            if not os.path.exists(file_path):
                print '文件夹',file_path,'不存在，重新建立'
                #os.mkdir(file_path)
                os.makedirs(file_path)
            #获得图片后缀
            file_suffix = os.path.splitext(img_url)[1]
            #拼接图片名（包含路径）
            filename = '{}{}{}{}'.format(file_path,os.sep,file_name,file_suffix)
            filename=filename.replace('.com','_com')
            if string.find(filename,'.')==-1:
                filename=filename+'.png'
            if string.find(filename,'?')!=-1:
                filename=filename[0:len(filename)-4]
#       m下载图片，并保存到文件夹中
            urllib.urlretrieve(img_url,filename=filename)
        except IOError as e:
            print '文件操作失败',e
        except Exception as e:
            print '错误 ：',e
    def close(self,reason):
        print "采集完成，共采集了["+str(self.num)+"]条数据 "
